package com.hlmc.spider;

import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;

@Service("zy2_spider")
public class Zy2_spider extends BaseProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setCharset("utf-8" + "");

    @Override
    public void process(Page page) {

        page.addTargetRequests(page.getHtml().xpath("//div[@class='nav']/ul/li/a/@href").all());

        if(page.getUrl().toString().contains("http://yongjiuzy.cc/?m=vod-type-id-"))
        {
            page.addTargetRequests(page.getHtml().xpath("//tr/td[1]/a/@href").all());
            page.addTargetRequests(page.getHtml().css("a.pagelink_a","href").all());
        }
        if(page.getUrl().toString().contains("http://yongjiuzy.cc/?m=vod-detail-id")){
            String image=page.getHtml().xpath("//div[@class='videoPic']/img/@src").toString();
            String category=page.getHtml().xpath("//div[@class='videoDetail']/li[6]/div[@class='left']/text()").toString().replace("栏目分类:","").replace(" ","");
            String title=page.getHtml().xpath("//div[@class='videoDetail']/li[1]/text()").toString().replace("影片名称:","").replace(" ","");
            String year=page.getHtml().xpath("//div[@class='videoDetail']/li[8]/div[@class='right']/text()").toString().replace("上映年份:","").replace(" ","");
            String url=page.getHtml().xpath("//div[@class='movievod']/ul/li/input/@value").all().toString().replace("[","").replace("]","").replace(" ","");
            if( (image == null)||(image.replace(" ", "").equals(""))){

                page.setSkip(true);
            }

            if (url != null&&!url.replace(" ", "").equals("")){
                page.putField("image",image);
                page.putField("title",title);
                page.putField("year",year);
                page.putField("url",url);
                page.putField("category",category);
            }

        }

    }
    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {

        // private  DS_pipeline ds_pipeline;
        Spider.create(new Zy2_spider()).addUrl("http://yongjiuzy.cc/")
                .addPipeline(new Zy2_pipeline())
                .thread(5).run();

    }

}